import java.io.FileWriter;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.junit.Test;

import com.bmth.alise.client.config.Configuration;
import com.bmth.alise.client.utils.RedisUtils;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.utils.UrlUtils;

public class d {

	public static void main(String[] args) {

		String canonicalizeUrl = UrlUtils.canonicalizeUrl("#", "http://skysfw.blog.51cto.com/926181/1880505#");

		System.out.println(canonicalizeUrl);
	}

	@Test
	public void t() {
		String shaHex = DigestUtils.shaHex("http://down.51cto.com/data/2262049");
		System.out.println(shaHex);
	}

	@Test
	public void dispather() {
		Configuration.init();
		JedisPool pool = RedisUtils.initPool();
		Jedis resource = pool.getResource();
		List<String> lrange = resource.lrange("queue_search_it", 0, 2000000);
		Set<String> set = new HashSet<>();
		for (String string : lrange) {
			set.add(string);
		}
		try {
			IOUtils.writeLines(set, "\n", new FileWriter("/Users/u01/set.txt"));
		} catch (Exception e) {
			e.printStackTrace();
		}

	}
}
